<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "DTD/xhtml1-transitional.dtd">















 
 
 
 
 
 
 
  
  


<html>
  <head>
    <script type="text/javascript" language="JavaScript">
    ORIGINAL_PAGE_PATH = "/appengine/docs/python/tools/uploadingdata.html";
    </script>
    
    
<meta http-equiv="content-type" content="text/html; charset=utf-8"/>
<title>Uploading Data - Google App Engine - Google Code</title>
<script type="text/javascript"><!--
(function(){function a(){this.t={};this.tick=function(c){this.t[c]=(new Date).getTime()};this.tick("start")}var b=new a;window.jstiming={Timer:a,load:b};if(window.external&&window.external.pageT)window.jstiming.pt=window.external.pageT;})();

var _tocPath_ = '/appengine/docs/_toc.ezt';
var codesite_token = null;
//--></script>
<link href="../../../../css/codesite.pack.04102009.css" type="text/css" rel="stylesheet"></link>
<script src="../../../../js/codesite_head.pack.04102009.js" type="text/javascript"></script>
<script type="text/javascript">CODESITE_CSITimer['load'].tick('bhs');</script>
<link rel="search" type="application/opensearchdescription+xml" title="Google Code" href="http://code.google.com/osd.xml" />

<!--[if IE]><link rel="stylesheet" type="text/css" href="../../../../css/iehacks.css" /><![endif]-->

    <link href="../../../../css/semantic_headers.css" rel="stylesheet" type="text/css" />
    <link href="../../../css/local_extensions.css" rel="stylesheet" type="text/css" />
  </head>

  <body class="gc-documentation">

    
    
    
</a>

<div id="gb">
 <span>
  
    <a id="lang-dropdown" class="dropdown" href="http://code.google.com" onclick="return false;"><img class="globeicon" src="../../../../images/globe2_small.png"/><span>English</span></a>
  
 </span>
</div>

<div class="gbh" style="left: 0pt;"></div>
<div class="gbh" style="right: 0pt;"></div>


<style type="text/css">
  #gc-topnav h1 {
    padding: 0 0 0 6px;
  }
</style>


<div id="gc-container">
<a name="top"></a>
<div id="skipto">
  <a href="#gc-pagecontent">Skip to page content</a>
  <a href="#gc-toc">Skip to main navigation</a>
</div>

<div id="gc-header">
  <div id="logo"><a href="http://code.google.com">
  
  
     <img src="../../../../images/cleardot.gif" height="1px" width="1px" alt="Google Code Home Page" id="gc-logo-img"/>
  
  
  </a></div>
  <div id="search">
    <div id="searchForm" class="searchForm">
      <form id="cse" action="http://www.google.com/cse" accept-charset="utf-8" class="gsc-search-box" onsubmit="executeGSearch(document.getElementById('gsearchInput').value); return false;">
      <noscript>
      <input type="hidden" name="cref" value="http://code.google.com/cse/googlecode-context.xml"/>
      </noscript>
        <table class="gsc-search-box" cellpadding="0" cellspacing="0">
          <tbody>
            <tr>
              <td class="gsc-input">
                <input id="gsearchInput" type="text" name="q" maxlength="2048" class="gsc-input" autocomplete="off" title="Google Code Search" style="width:345px"/>
              </td>
              <td class="gsc-search-button">
                <div id="cs-searchresults" onclick="event.cancelBubble = true;"></div>
                <input title="Search" id="gsearchButton" class="gsc-search-button" name="sa" value="Search" type="submit"/>
              </td>
            </tr>
            <tr>
              <td colspan="2" class="greytext">e.g. "templates" or "datastore"</td>
            </tr>
          </tbody>
        </table>
      </form>
    </div> <!-- end searchForm -->
  </div> <!-- end search -->
</div> <!-- end gc-header -->


<div id="codesiteContent">

<a name="gc-topnav-anchor"></a>
<div id="gc-topnav">
  <h1>Google App Engine</h1>
  <ul id="docs" class="gc-topnav-tabs">

    <li id="home_link">
      <a href="../../../index.html" title="Google App Engine home page">Home</a>
    </li>
  
    <li id="docs_link">
      <a href="../../index.html" class="selected" title="Official Google App Engine documentation">Docs</a>
    </li>
  
    <li id="faq_link">
      <a href="../../../kb/index.html" title="Answers to frequently asked questions about Google App Engine">FAQ</a>
    </li>
  
    <li id="articles_link">
      <a href="../../../articles/index.html" title="Focused articles and tutorials for Google App Engine developers">Articles</a>
    </li>
  
    <li>
      <a href="http://googleappengine.blogspot.com/" title="Official Google App Engine blog">Blog</a>
    </li>
  
    <li>
      <a href="../../../community.html" title="Community home for Google App Engine">Community</a>
    </li>
  
    <li>
      <a href="../../../terms.html" title="Google App Engine terms of service">Terms</a>
    </li>
  
    <li>
      <a href="../../../downloads.html" title="Download Google App Engine">Download</a>
    </li>
  

  </ul>
</div> <!-- end gc-topnav -->

    <div class="g-section g-tpl-170">

      <div class="g-unit g-first" id="gc-toc">
        <ul>
  <li><a href="../../../downloads.html">Downloads</a></li>
  <li><a href="http://code.google.com/status/appengine">System Status</a></li>
  <li><a href="http://code.google.com/p/googleappengine/issues/list">Issue Tracker</a></li>
</ul>
<div class="line"></div>
<ul>
  <li><h2>Getting Started</h2>
    <ul>
      <li><a href="../../whatisgoogleappengine.html">What Is Google App Engine?</a></li>
      <li><a href="../../java/gettingstarted/index.html">Java</a>
        <ul>
              <li><a href="../../java/gettingstarted/introduction.html">Introduction</a></li>
    <li><a href="../../java/gettingstarted/installing.html">Installing the Java SDK</a></li>
    <li><a href="../../java/gettingstarted/creating.html">Creating a Project</a></li>
    <li><a href="../../java/gettingstarted/usingusers.html">Using the Users Service</a></li>
    <li><a href="../../java/gettingstarted/usingjsps.html">Using JSPs</a></li>
    <li><a href="../../java/gettingstarted/usingdatastore.html">Using the Datastore with JDO</a></li>
    <li><a href="../../java/gettingstarted/staticfiles.html">Using Static Files</a></li>
    <li><a href="../../java/gettingstarted/uploading.html">Uploading Your Application</a></li>

        </ul>
      </li>
      <li><a href="../gettingstarted/index.html">Python</a>
        <ul>
            <li><a href="../gettingstarted/introduction.html">Introduction</a></li>
  <li><a href="../gettingstarted/devenvironment.html">The Development Environment</a></li>
  <li><a href="../gettingstarted/helloworld.html">Hello, World!</a></li>
  <li><a href="../gettingstarted/usingwebapp.html">Using the webapp Framework</a></li>
  <li><a href="../gettingstarted/usingusers.html">Using the Users Service</a></li>
  <li><a href="../gettingstarted/handlingforms.html">Handling Forms With webapp</a></li>
  <li><a href="../gettingstarted/usingdatastore.html">Using the Datastore</a></li>
  <li><a href="../gettingstarted/templates.html">Using Templates</a></li>
  <li><a href="../gettingstarted/staticfiles.html">Using Static Files</a></li>
  <li><a href="../gettingstarted/uploading.html">Uploading Your Application</a></li>

        </ul>
      </li>
    </ul>
  </li>
</ul>
<div class="line"></div>
<ul>
  <li><h2>Java <sup class="new">Early Look</sup></h2>
    <ul>
          <li><a href="../../java/overview.html">Overview</a></li>
    <li><a href="../../java/runtime.html">Servlet Environment</a></li>
    <li><a href="../../java/datastore/index.html">Storing Data</a>
      <ul>
            <li><a href="../../java/datastore/overview.html">Overview</a></li>
    <li><a href="../../java/datastore/usingjdo.html">Using JDO</a></li>
    <li><a href="../../java/datastore/dataclasses.html">Defining Data Classes</a></li>
    <li><a href="../../java/datastore/creatinggettinganddeletingdata.html">Creating, Getting and Deleting Data</a></li>
    <li><a href="../../java/datastore/queriesandindexes.html">Queries and Indexes</a></li>
    <li><a href="../../java/datastore/transactions.html">Transactions</a></li>
    <li><a href="../../java/datastore/relationships.html">Relationships</a></li>
    <li><a href="../../java/datastore/usingjpa.html">Using JPA</a></li>
    <li><a href="../../java/javadoc/com/google/appengine/api/datastore/package-summary.html">Low-level API</a></li>

      </ul>
    </li>
    <li><a href="../../java/apis.html">Services</a>
      <ul>
        <li><a href="../../java/memcache/index.html">Memcache</a>
          <ul>
                <li><a href="../../java/memcache/overview.html">Overview</a></li>
    <li><a href="../../java/memcache/usingjcache.html">Using JCache</a></li>
    <li><a href="../../java/javadoc/com/google/appengine/api/memcache/package-summary.html">Low-level API</a></li>

          </ul>
        </li>
        <li><a href="../../java/urlfetch/index.html">URL Fetch</a>
          <ul>
                <li><a href="../../java/urlfetch/overview.html">Overview</a></li>
    <li><a href="../../java/urlfetch/usingjavanet.html">Using java.net</a></li>
    <li><a href="../../java/javadoc/com/google/appengine/api/urlfetch/package-summary.html">Low-level API</a></li>

          </ul>
        </li>
        <li><a href="../../java/mail/index.html">Mail</a>
          <ul>
                <li><a href="../../java/mail/overview.html">Overview</a></li>
    <li><a href="../../java/mail/usingjavamail.html">Using JavaMail</a></li>
    <li><a href="../../java/javadoc/com/google/appengine/api/mail/package-summary.html">Low-level API</a></li>

          </ul>
        </li>
        <li><a href="../../java/images/index.html">Images</a>
          <ul>
                <li><a href="../../java/images/overview.html">Overview</a></li>
    <li><a href="../../java/javadoc/com/google/appengine/api/images/package-summary.html">API Reference</a></li>

          </ul>
        </li>
        <li><a href="../../java/users/index.html">Google Accounts</a>
          <ul>
                <li><a href="../../java/users/overview.html">Overview</a></li>
    <li><a href="../../java/javadoc/com/google/appengine/api/users/package-summary.html">API Reference</a></li>

          </ul>
        </li>
      </ul>
    </li>
    <li><a href="../../java/javadoc/index.html">Javadoc Reference</a></li>
    <li><a href="../../java/jrewhitelist.html">JRE Class White List</a></li>
    <li><a href="../../java/config/index.html">Configuration</a>
      <ul>
            <li><a href="../../java/config/webxml.html">Deployment Descriptor</a></li>
    <li><a href="../../java/config/appconfig.html">App Config</a></li>
    <li><a href="../../java/config/indexconfig.html">Index Config</a></li>
    <li><a href="../../java/config/cron.html">Scheduled Tasks</a></li>

      </ul>
    </li>
    <li><a href="../../java/tools/index.html">Tools</a>
      <ul>
            <li><a href="../../java/tools/devserver.html">Development Server</a></li>
    <li><a href="../../java/tools/uploadinganapp.html">Uploading and Managing</a></li>
    <li><a href="../../java/tools/eclipse.html">Google Plugin for Eclipse</a></li>
    <li><a href="../../java/tools/ant.html">Using Apache Ant</a></li>

      </ul>
    </li>
    <li><a href="../../java/howto/index.html">How-To</a>
      <ul>
              <li><a href="../../java/howto/unittesting.html">Unit Testing</a></li>

      </ul>
    </li>

    </ul>
  </li>
</ul>
<div class="line"></div>
<ul>
  <li><h2>Python</h2>
    <ul>
          <li><a href="../overview.html">Overview</a></li>
    <li><a href="../runtime.html">CGI Environment</a></li>
    <li><a href="../datastore/index.html">Storing Data</a>
      <ul>
             <li><a href="../datastore/overview.html">Overview</a></li>
     <li><a href="../datastore/entitiesandmodels.html">Entities and Models</a></li>
     <li><a href="../datastore/creatinggettinganddeletingdata.html">Creating, Getting and Deleting Data</a></li>
     <li><a href="../datastore/keysandentitygroups.html">Keys and Entity Groups</a></li>
     <li><a href="../datastore/queriesandindexes.html">Queries and Indexes</a></li>
     <li><a href="../datastore/transactions.html">Transactions</a></li>
     <li><a href="../datastore/typesandpropertyclasses.html">Types and Property Classes</a></li>
     <li><a href="../datastore/gqlreference.html">GQL Reference</a></li>

     <li><span class="tlw-title tlw-expanded">Reference</span>
       <ul>
         <li><a href="../datastore/modelclass.html">Model</a></li>
         <li><a href="../datastore/expandoclass.html">Expando</a></li>
         <li><a href="../datastore/polymodelclass.html">PolyModel</a></li>
         <li><a href="../datastore/propertyclass.html">Property</a></li>
         <li><a href="../datastore/queryclass.html">Query</a></li>
         <li><a href="../datastore/gqlqueryclass.html">GqlQuery</a></li>
         <li><a href="../datastore/keyclass.html">Key</a></li>
         <li><a href="../datastore/functions.html">Functions</a></li>
         <li><a href="../datastore/exceptions.html">Exceptions</a></li>
       </ul>
     </li>

      </ul>
    </li>
    <li><a href="../apis.html">Services</a>
      <ul>
        <li><a href="../memcache/index.html">Memcache</a>
          <ul>
                 <li><a href="../memcache/overview.html">Overview</a></li>
      <li><a href="../memcache/usingmemcache.html">Using Memcache</a></li>
     <li><span class="tlw-title tlw-expanded">Reference</span>
       <ul>
         <li><a href="../memcache/clientclass.html">Client</a></li>
         <li><a href="../memcache/functions.html">Functions</a></li>
       </ul>
     </li>

          </ul>
        </li>
        <li><a href="../urlfetch/index.html">URL Fetch</a>
          <ul>
                 <li><a href="../urlfetch/overview.html">Overview</a></li>
     <li><span class="tlw-title tlw-expanded">Reference</span>
       <ul>
         <li><a href="../urlfetch/fetchfunction.html">The fetch Function</a></li>
         <li><a href="../urlfetch/responseobjects.html">Response Objects</a></li>
         <li><a href="../urlfetch/exceptions.html">Exceptions</a></li>
       </ul>
     </li>

          </ul>
        </li>
        <li><a href="../mail/index.html">Mail</a>
          <ul>
                 <li><a href="../mail/overview.html">Overview</a></li>
     <li><a href="../mail/sendingmail.html">Sending Mail</a></li>
     <li><a href="../mail/attachments.html">Attachments</a></li>
     <li><span class="tlw-title tlw-expanded">Reference</span>
       <ul>
         <li><a href="../mail/emailmessageclass.html">EmailMessage</a></li>
         <li><a href="../mail/emailmessagefields.html">Message Fields</a></li>
         <li><a href="../mail/functions.html">Functions</a></li>
         <li><a href="../mail/exceptions.html">Exceptions</a></li>
       </ul>
     </li>

          </ul>
        </li>
        <li><a href="../images/index.html">Images</a>
          <ul>
                 <li><a href="../images/overview.html">Overview</a></li>
     <li><a href="../images/installingPIL.html">Installing PIL</a></li>
     <li><a href="../images/usingimages.html">Using the Images API</a></li>
     <li><span class="tlw-title tlw-expanded">Reference</span>
       <ul>
         <li><a href="../images/imageclass.html">Image</a></li>
         <li><a href="../images/functions.html">Functions</a></li>
         <li><a href="../images/exceptions.html">Exceptions</a></li>
       </ul>
     </li>

          </ul>
        </li>
        <li><a href="../users/index.html">Google Accounts</a>
          <ul>
                 <li><a href="../users/overview.html">Overview</a></li>
     <li><a href="../users/userobjects.html">User Objects</a></li>
     <li><a href="../users/loginurls.html">Login URLs</a></li>
     <li><a href="../users/adminusers.html">Admin Users</a></li>

     <li><span class="tlw-title tlw-expanded">Reference</span>
       <ul>
         <li><a href="../users/userclass.html">User</a></li>
         <li><a href="../users/functions.html">Functions</a></li>
         <li><a href="../users/exceptions.html">Exceptions</a></li>
       </ul>
     </li>

          </ul>
        </li>
      </ul>
    </li>
    <li><a href="../config/index.html">Configuration</a>
      <ul>
            <li><a href="../config/appconfig.html">App Config</a></li>
    <li><a href="../config/indexconfig.html">Index Config</a></li>
    <li><a href="../config/cron.html">Scheduled Tasks</a></li>

      </ul>
    </li>
    <li><a href="index.html">Tools</a>
      <ul>
            <li><a href="devserver.html">Development Server</a></li>
    <li><a href="uploadinganapp.html">Uploading and Managing</a></li>
    <li><a href="uploadingdata.html">Uploading Data</a></li>
    <li><a href="webapp/index.html">webapp Framework</a>
      <ul>
             <li><a href="webapp/overview.html">Overview</a></li>
     <li><a href="webapp/running.html">Running the Application</a></li>
     <li><a href="webapp/requesthandlers.html">Request Handlers</a></li>
     <li><a href="webapp/requestdata.html">Request Data</a></li>
     <li><a href="webapp/buildingtheresponse.html">Building the Response</a></li>
     <li><a href="webapp/redirects.html">Redirects, Headers and Status Codes</a></li>
     

     <li><span class="tlw-title tlw-expanded">Reference</span>
       <ul>
         <li><a href="webapp/requestclass.html">Request</a></li>
         <li><a href="webapp/responseclass.html">Response</a></li>
         <li><a href="webapp/requesthandlerclass.html">RequestHandler</a></li>
         <li><a href="webapp/wsgiapplicationclass.html">WSGIApplication</a></li>
         <li><a href="webapp/utilmodule.html">Utility Functions</a></li>
         
       </ul>
     </li>

      </ul>
    </li>
    <li><a href="libraries.html">Third-party Libraries</a></li>

      </ul>
    </li>
    <li><a href="../howto/index.html">How-To</a>
      <ul>
              <li><a href="../howto/usinggdataservices.html">Google Data Services</a></li>

      </ul>
    </li>

    </ul>
  </li>
</ul>
<div class="line"></div>
<ul>
  <li><h2>Managing Your App</h2>
    <ul>
      <li><a href="../../theadminconsole.html">The Admin Console</a></li>
      <li><a href="../../quotas.html">Quotas</a></li>
      <li><a href="../../billing.html">Billing</a></li>
    </ul>
  </li>
</ul>
<div class="line"></div>
<ul>
  <li><h2>Resources</h2>
    <ul>
      <li><a href="../../../kb/index.html">FAQ</a></li>
      <li><a href="../../../articles/index.html">Articles</a></li>
      <li><a href="http://appengine-cookbook.appspot.com/">Cookbook</a></li>
      <li><a href="http://appgallery.appspot.com/">App Gallery</a></li>
      <li><a href="http://code.google.com/p/googleappengine/">SDK Code</a></li>
      <li><a href="http://code.google.com/p/google-app-engine-samples/">Sample Apps Code</a></li>
      <li><a href="../../../community.html">Discussion Groups</a></li>
    </ul>
  </li>
</ul>
<div class="line"></div>
<ul>
  <li><a href="../../roadmap.html">Product Roadmap</a></li>
  <li><a href="http://code.google.com/p/googleappengine/wiki/SdkReleaseNotes">Release Notes</a></li>
  <li><a href="../../revision_history.html">Revision History</a></li>
</ul>

        <a class="hidden" href="#gc-topnav-anchor">More Google App Engine resource links</a>
      </div>

      <div class="g-unit" id="gc-pagecontent">
        <h1 class="page_title">Uploading Data</h1>



<p>The <code>appcfg.py</code> tool can upload and download data to and from your application's datastore.  With just a little bit of set-up, you can upload new datastore entities from CSV files, and download entity data into CSV files.  Most spreadsheet applications can export CSV files, making it easy for non-developers and other application to produce data that can be imported into your application.  You can customize the upload and download logic to use different kinds of files, or do other data processing.</p>

<ul>
  <li><a href="#Setting_Up_remote_api">Setting Up remote_api</a></li>
  <li><a href="#Installing_Sqlite">Installing Sqlite (recommended)</a></li>
  <li><a href="#Creating_Loader_Classes">Creating Loader Classes</a></li>
  <li><a href="#Preparing_Your_Data">Preparing Your Data</a></li>
  <li><a href="#Uploading_the_Data_to_App_Engine">Uploading the Data to App Engine</a></li>
  <li><a href="#Loading_Data_Into_the_Development_Server">Loading Data Into the Development Server</a></li>
  <li><a href="#Creating_Exporter_Classes">Creating Exporter Classes</a></li>
  <li><a href="#Downloading_Data_from_App_Engine">Downloading Data from App Engine</a></li>
  <li><a href="#Command_Line_Arguments">Command Line Arguments</a></li>
</ul>


<h2 id="Setting_Up_remote_api">Setting Up remote_api</h2>

<p>The data loader feature communicates with your application running on App Engine using remote_api, a request handler included with the App Engine Python library that allows remote applications with the proper credentials to access the datastore remotely.  To use remote_api, you must map a URL to it in your <code>app.yaml</code> file.</p>

<p>Edit your <code>app.yaml</code>, and add the following lines to the <code>handlers:</code> section:</p>

<pre>
- url: /remote_api
  script: $PYTHON_LIB/google/appengine/ext/remote_api/handler.py
  login: admin
</pre>

<p>This maps the remote_api request handler to the URL <code>/remote_api</code> for your application.  You can use any URL you like.  Access to this URL is restricted to users are administrators for the application.</p>

<p>Update your app to install the new <code>app.yaml</code> and the remote_api URL:</p>

<pre>
appcfg.py update
</pre>


<h2 id="Installing_Sqlite">Installing Sqlite (recommended)</h2>

<p><code>appcfg.py</code> uses a data file to keep track of its progress while uploading or downloading data, in case the process is interrupted and needs to be resumed.  This feature requires the Sqlite 3 library to manage the progress file.  If you do not have Sqlite 3 installed on your system, see <a href="http://www.sqlite.org/download.html">the Sqlite web site</a> for information on downloading and installing Sqlite 3.</p>

<p class="note"><b>Note:</b> If you installed Python on your system by compiling it from source code, you will need to recompile Python with Sqlite support after installing Sqlite.</p>

<p>You can test if Sqlite is installed for Python by entering the following command:</p>

<pre>python -c 'import sqlite3'</pre>

<p>If the command does nothing, then Sqlite 3 is installed.  If Sqlite 3 is not installed, it prints: <code>ImportError: No module named sqlite3</code></p>

<p>If you do not wish to install Sqlite 3, you can disable the use of a progress file.  To do so, give <code>appcfg.py</code> this argument: <code>--db_filename=skip</code>  If you disable the progress file and an upload or download is interrupted, the process must be restarted from the beginning.</p>


<h2 id="Creating_Loader_Classes">Creating Loader Classes</h2>

<p>To upload data, you must tell <code>appcfg.py</code> how to translate each row in the data file to a datastore entity.  You do this using a file of Python code.  The file imports or defines the Model classes for the entities being created, defines a loader class for each kind you wish to import, and declares the available loader classes in a global variable.</p>

<p>For example, say you have a Model class named <code>Album</code> defined in a file named <code>models.py</code> (which is in your PYTHONPATH, such as the directory where you'll run the tool) that resembles the following:</p>

<pre class="prettyprint">
from google.appengine.ext import db

class Album(db.Model):
  artist = db.StringProperty()
  title = db.StringProperty()
  publication_date = db.DateProperty()
  length_in_minutes = db.IntegerProperty()
</pre>

<p>You wish to import a CSV file that has the columns in the following order: title, artist, publication date, and length in minutes.  The CSV file contains string representations for each of these values.</p>

<p>The following is a loader class file for this data:</p>

<pre class="prettyprint">
import datetime
from google.appengine.ext import db
from google.appengine.tools import bulkloader
import models

class AlbumLoader(bulkloader.Loader):
  def __init__(self):
    bulkloader.Loader.__init__(self, 'Album',
                               [('title', str),
                                ('artist', str),
                                ('publication_date',
                                 lambda x: datetime.datetime.strptime(x, '%m/%d/%Y').date()),
                                ('length_in_minutes', int)
                               ])

loaders = [AlbumLoader]
</pre>

<p>The bulk loader tool looks for a global variable in the loader class file named <code>loaders</code>, whose value is a list of loader classes to use.  In this example, the tool loads the <code>AlbumLoader</code> class for loading entities of the kind <code>Album</code>.</p>

<p>The loader class defines an <code>__init__()</code> method that calls the <code>__init__()</code> on the <code>Loader</code> class.  The first argument is <code>self</code>, the loader class instance.  The second argument is the name of the datastore kind as a string, in this case <code>'Album'</code>.  The third argument is a sequence of tuples, where each tuple contains the name of the property (a string) and a conversion function.  The conversion function must take a string and return one of <a href="../datastore/typesandpropertyclasses.html">the datastore value types</a>.</p>

<p>In this example, the <code>'title'</code> and <code>'artist'</code> properties both take string values, so the conversion function is <code>str</code>, the string constructor.  The <code>'length_in_minutes'</code> property takes an integer, and the <code>int</code> constructor takes a string and converts it to an integer value.</p>

<p>For the <code>'publication_date'</code> property, the model needs a <code>datetime</code> value.  In this case, we know (from our data file) that publication dates are represented in the form <code>mm/dd/yyyy</code>.  The conversion function is a Python lambda expression (a short function) that takes a string, then passes it to <code>datetime.datetime.strptime()</code> with a pattern to parse the value to a <code>datetime.datetime</code>, then calls its <code>date()</code> method to get the final <code>datetime.date</code> value.</p>

<p>If any conversion function raises an exception or fails to return a value that meets the requirements of the Model class, processing of the data file stops and reports an error.  If you are using a progress file (with Sqlite, as described above), you can fix the CSV file (or the code, as needed), then re-run the upload to continue starting from the row where the error occurred.</p>

<p>If the name of the file containing the loader class definition is <code>album_loader.py</code>, you would give <code>appcfg.py upload_data</code> the following argument: <code>--config_file=album_loader.py</code></p>




<h2 id="Preparing_Your_Data">Preparing Your Data</h2>

<p><code>appcfg.py upload_data</code> accepts data in the form of a file in the CSV (Comma Separated Values) format, a simple text file representing a table of values, with one row per line and columns delimited by commas (<code>,</code>).  If a value contains one or more commas, the value is surrounded by double quotes (<code>&quot;</code>).  If a value contains a double quote, the double quote appears twice (<code>&quot;&quot;</code>).  The tool uses the <a href="http://www.python.org/doc/2.5.2/lib/module-csv.html">csv</a> module from the Python standard library to parse the data file.</p>

<p>The file must use the UTF-8 encoding for the text data.  UTF-8 encoding is compatible with ASCII encoding, so any spreadsheet application that saves CSV as ASCII will also work.</p>

<p>Most spreadsheet applications can export a sheet as a CSV file.  To export a sheet as CSV from <a href="http://docs.google.com/">Google Spreadsheets</a>, select the <b>File</b> menu &gt; <b>Export</b> &gt; <b>.csv Sheet Only</b>, then save the file from the browser window that opens.</p>

<p>If the name of the CSV file to load is <code>album_data.csv</code>, you would give <code>appcfg.py upload_data</code> the following argument: <code>--filename=album_data.csv</code>

<p>If the CSV file's first row is not data (such as if it's a header row), use the following option to skip the first row: <code>--has_header</code></p>


<h2 id="Uploading_the_Data_to_App_Engine">Uploading the Data to App Engine</h2>

<p>To start the data upload, run <code>appcfg.py upload_data</code> with the appropriate arguments:</p>

<pre>
appcfg.py upload_data --config_file=album_loader.py --filename=album_data.csv --kind=Album <i>&lt;app-directory&gt;</i></i>
</pre>

<p>If you are using a Google Apps domain name and need appcfg.py to sign in using an account on that domain, you must specify the <code>--auth_domain=...</code> option, whose value is your domain name.</p>

<p>If the transfer is interrupted, you can resume the transfer from where it left off using the <code>--db_filename=...</code> argument.  The value is the name of the progress file created by the tool, which is either a name you provided with the <code>--db_filename</code> argument when you started the transfer, or a default name that includes a timestamp.  This assumes you have <code>sqlite3</code> installed, and did not disable the progress file with <code>--db_filename=skip</code>.</p>


<h2 id="Loading_Data_Into_the_Development_Server">Loading Data Into the Development Server</h2>

<p>If you'd like to test how your data works with the app before uploading it, you can load it into the development server.  Use the <code>--url</code> option to point the tool at the development server URL.  For example:</p>

<pre>
appcfg.py upload_data --config_file=album_loader.py --filename=album_data.csv --kind=Album --url=http://localhost:8080/remote_api <i>&lt;app-directory&gt;</i>
</pre>


<h2 id="Creating_Exporter_Classes">Creating Exporter Classes</h2>

<p>To download data, you must tell <code>appcfg.py</code> how to export datastore entities to a file.  Similar to uploading and loader classes, you do this with a file of Python code.  The file imports or defines the Model classes, defines an exporter class for each kind you wish to export, and declares the available exporter classes in a global variable.</p>

<p>Continuing the example with the <code>Album</code> Model class defined above, the following file defines an exporter that produces a CSV file in the same format used by the loader class:</p>

<pre class="prettyprint">
from google.appengine.ext import db
from google.appengine.tools import bulkloader
import models

class AlbumExporter(bulkloader.Exporter):
  def __init__(self):
    bulkloader.Exporter.__init__(self, 'Album',
                                 [('title', str, None),
                                  ('artist', str, None),
                                  ('publication_date', str, None),
                                  ('length_in_minutes', str, None)
                                 ])

exporters = [AlbumExporter]
</pre>

<p>The exporter class is similar to the loader class.  The exporter class defines an <code>__init__()</code> method that calls the <code>__init__()</code> on the <code>Exporter</code> class.   The first argument is <code>self</code>, the exporter instance.  The second argument is the name of the datastore kind being exported, a string.  The third argument is a sequence of tuples, one for each entity property being exported.</p>

<p>Each tuple has 3 elements: the property name, a function that takes the property value and converts it to a <code>str</code>, and a default value if the entity does not have the property set.  If the default value is <code>None</code>, the exporter raises an exception if it attempts to export an entity that does not have the property.</p>

<p>The CSV file created by the exporter includes one row for each entity, and one column for each property mentioned in the sequence.  The columns are in the order they appear in the sequence.</p>



<p class="note"><b>Tip:</b> You can define loader classes and exporter classes in the same file, and use the same arguments for <code>appcfg.py upload_data</code> and <code>appcfg.py download_data</code> to transfer data in both directions.</p>


<h2 id="Downloading_Data_from_App_Engine">Downloading Data from App Engine</h2>

<p>To start a data download, run <code>appcfg.py download_data</code> with the appropriate arguments:</p>

<pre>
appcfg.py download_data --config_file=album_loader.py --filename=album_data_archive.csv --kind=Album <i>&lt;app-directory&gt;</i></i>
</pre>

<p>If you are using a Google Apps domain name and need appcfg.py to sign in using an account on that domain, you must specify the <code>--auth_domain=...</code> option, whose value is your domain name.</p>

<p>If the transfer is interrupted, you can resume the transfer from where it left off using the <code>--db_filename=...</code> and <code>--result_db_filename=...</code> arguments.  These arguments are the names of the progress file and the results file created by the tool, which are either names you provided with the arguments when you started the transfer, or default names that include a timestamp.  This assumes you have <code>sqlite3</code> installed, and did not disable progress files with <code>--db_filename=skip</code>.</p>


<h2 id="Command_Line_Arguments">Command Line Arguments</h2>

<p>The <code>appcfg.py upload_data</code> command accepts the following arguments.  See also the other options <code>appcfg.py</code> accepts for all actions, listed in <a href="uploadinganapp.html#Command_Line_Arguments">Uploading and Managing a Python App: Command-Line Arguments</a>.</p>

<dl>
  <dt><code>appcfg.py upload_data <i>[options]</i></code> <i>&lt;app-directory&gt;</i></dt>
  <dd>
    <dl>
      <dt><code>--filename=<i>...</i></code></dt>
      <dd>
        <p>Required.  The path to the CSV data file to load.</p>
      </dd>

      <dt><code>--kind=<i>...</i></code></dt>
      <dd>
        <p>Required.  The name of the datastore kind to use for creating new entities.</p>
      </dd>

      <dt><code>--config_file=<i>...</i></code></dt>
      <dd>
        <p>Required.  A Python source file that imports or defines Model classes for the kinds of entities that an upload might create, as well as Loader classes for each kind.  <code>appcfg.py upload_data</code> provides the <code>Loader</code> base class in the local namespace when evaluating this file.</p>
      </dd>

      <dt><code>--loader_opts=<i>...</i></code></dt>
      <dd>
        <p>An option to pass to the Loader class's <code>initialize()</code> method.  You can implement this method to pass arguments to your Loader classes.</p>
      </dd>

      <dt><code>--log_file=<i>...</i></code></dt>
      <dd>
        <p>The name of the file to write logging information about the upload.  The default is to create a file named <code>bulkloader-log-<i>timestamp</i></code> in the current working directory (where <code><i>timestamp</i></code> is the time the tool is run).</p>
      </dd>

      <dt><code>--auth_domain=<i>...</i></code></dt>
      <dd>
        <p>The name of the authorization domain of the account to use to contact remote_api.  If you're using a Google Apps domain and need appcfg.py to sign in using a Google Apps account, specify your domain name with this option.</p>
      </dd>

      <dt><code>--num_threads=<i>#</i></code></dt>
      <dd>
        <p>The number of threads to spawn to upload new entities in parallel.  The default is 10.</p>
      </dd>

      <dt><code>--batch_size=<i>#</i></code></dt>
      <dd>
        <p>The number of entities to create with each remote_api call.  For large entities, use a small batch size to limit the amount of data posted per batch.  The default is 10.</p>
      </dd>

      <dt><code>--bandwidth_limit=<i>#</i></code></dt>
      <dd>
        <p>The maximum total number of bytes per second that all threads should send.  Bursts may exceed this maximum, but the overall bandwidth will be less than this amount.  The default is 250,000 bytes per second.</p>
      </dd>

      <dt><code>--rps_limit=<i>#</i></code></dt>
      <dd>
        <p>The maximum total number of records per second that all threads should send.  The default is 20.</p>
      </dd>

      <dt><code>--http_limit=<i>#</i></code></dt>
      <dd>
        <p>The maximum total number of HTTP requests per second that all threads should send.  The default is 7.5 per second (15 per 2 seconds).</p>
      </dd>

      <dt><code>--db_filename=<i>...</i></code></dt>
      <dd>
        <p>The filename to use for the progress file for this run.  If not specified, this is named <code>bulkloader-progress-<i>timestamp</i></code>, where <code><i>timestamp</i></code> represents the time the command is run.  If this argument is specified with the value <code>skip</code>, the upload will not use a progress file.</p>
      </dd>

      <dt><code>--has_header</code></dt>
      <dd>
        <p>If present, skips the first row in the CSV file, assuming it's a header row.</p>
      </dd>

      <dt><code>--app_id=<i>...</i></code></dt>
      <dd>
        <p>The application ID, if different from the application ID specified in the app's <code>app.yaml</code> file.</p>
      </dd>

      <dt><code>--url=<i>...</i></code></dt>
      <dd>
        <p>The URL of the remote_api handler to use to connect to the datastore.  By default, this is derived from the app's <code>app.yaml</code> file.</p>
      </dd>

    </dl>
  </dd>

  <dt><code>appcfg.py download_data <i>[options]</i></code> <i>&lt;app-directory&gt;</i></dt>
  <dd>
    <p>The arguments for <code>download_data</code> are the same as those for <code>upload_data</code>.  The <code>--filename</code> argument specifies the file to write to, instead of the file to read from.  This file is overwritten when starting a new download, and appended to when continuing an interrupted download.</p>
    <p><code>download_data</code> also supports the following argument:</p>
    <dl>
      <dt><code>--result_db_filename=<i>...</i></code></dt>
      <dd>
        <p>The filename to use for the results file for this run.  The results file stores the exported data until the export is finished, so the final output file can be sorted by key.  If not specified, this is named <code>bulkloader-results-<i>timestamp</i></code>, where <code><i>timestamp</i></code> represents the time the command is run.  If this argument is specified with the value <code>skip</code>, the upload will not use a results file.</p>
      </dd>
    </dl>
  </dd>

</dl>


      </div><!-- end gc-pagecontent -->
   </div><!-- end gooey wrapper -->

    </div> <!-- end codesite content -->

<div id="gc-footer" dir="ltr">
  <div class="text">
    
      <div class="notice"><div id="notice" style="text-align: center; border: 1em 0em 1em 0em">
  Except as otherwise <a
  href="http://code.google.com/policies.html#restrictions">noted</a>,
  the content of this page is licensed under the <a rel="license"
  href="http://creativecommons.org/licenses/by/2.5/">Creative Commons
  Attribution 2.5 License</a>, and code samples are licensed under the
  <a rel="license" href="http://www.apache.org/licenses/LICENSE-2.0">Apache
  2.0 License</a>.
<!-- <rdf:RDF xmlns="http://web.resource.org/cc/" 
              xmlns:dc="http://purl.org/dc/elements/1.1/"
              xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
  <Work rdf:about="">
    <license rdf:resource="http://creativecommons.org/licenses/by/2.5/" />
  </Work>
  <License rdf:about="http://creativecommons.org/licenses/by/2.5/">
    <permits rdf:resource="http://web.resource.org/cc/Reproduction"/>
    <permits rdf:resource="http://web.resource.org/cc/Distribution"/>
    <requires rdf:resource="http://web.resource.org/cc/Notice"/>
    <requires rdf:resource="http://web.resource.org/cc/Attribution"/>
    <permits rdf:resource="http://web.resource.org/cc/DerivativeWorks"/>
  </License>
</rdf:RDF> -->
</div>
Java is a registered trademark of Sun Microsystems, Inc.</div>
    
    &copy;2009 Google -
    <a href="http://code.google.com">Code Home</a> -
    <a href="http://www.google.com/accounts/TOS">Terms of Service</a> -
    <a href="http://www.google.com/privacy.html">Privacy Policy</a> -
    <a href="http://code.google.com/more">Site Directory</a>
    <br /> <br />
    Google Code offered in:
    <a href="http://code.google.com/intl/en">English</a> -
    <a href="http://code.google.com/intl/es">Español</a> -
    <a href="http://code.google.com/intl/ja">日本語</a> -
    <a href="http://code.google.com/intl/ko">한국어</a> -
    <a href="http://code.google.com/intl/pt-BR">Português</a> -
    <a href="http://code.google.com/intl/ru">Pусский</a> -
    <a href="http://code.google.com/intl/zh-CN">中文(简体)</a> -
    <a href="http://code.google.com/intl/zh-TW">中文(繁體)</a>
  </div>
</div><!-- end gc-footer -->

</div><!-- end gc-containter -->

<script type="text/javascript">CODESITE_CSITimer['load'].tick('ats');</script>
<script src="../../../../js/codesite_tail.pack.04102009.js" type="text/javascript"></script>






  </body>
</html>


